import asyncio
from crawl4ai import (
    AsyncWebCrawler,
    CrawlerRunConfig,
    CacheMode,
    CrawlResult,
)
from crawl4ai.content_filter_strategy import PruningContentFilter
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator

async def main():
    md_generator = DefaultMarkdownGenerator(
        content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed"),
    )
    config = CrawlerRunConfig(
        cache_mode = CacheMode.BYPASS,
        markdown_generator=md_generator,
    )
    async with AsyncWebCrawler() as crawler:
        result:CrawlResult = await crawler.arun(
            url="https://www.163.com",
            config=config,
        )

        print(f"Raw Markdown length: {len(result.markdown.raw_markdown)}")
        print(f"Fit Markdown length: {len(result.markdown.fit_markdown)}")

if __name__ == "__main__":
    asyncio.run(main())